{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import keras\n",
    "from keras.datasets import mnist\n",
    "from keras.utils import to_categorical\n",
    "from keras.layers import Conv2D=================================================================================================================================================================================================================================================================, Input, GlobalAveragePooling2D,Dense, Activation, Flatten, BatchNormalization\n",
    "# from keras.applications.mobilenet import relu6, DepthwiseConv2D\n",
    "from keras.layers import DepthwiseConv2D\n",
    "from keras import Model\n",
    "from keras import optimizers\n",
    "# from keras import activations"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://s3.amazonaws.com/img-datasets/mnist.npz\n",
      "11493376/11490434 [==============================] - 25s 2us/step\n",
      "(60000, 10)\n",
      "(10000, 10)\n"
     ]
    }
   ],
   "source": [
    "(train_x, train_y), (test_x, test_y) = mnist.load_data()\n",
    "train_x = train_x.reshape((60000,28,28,1)) / 255.0\n",
    "test_x = test_x.reshape((10000,28,28,1)) / 255.0\n",
    "\n",
    "train_y = keras.utils.to_categorical(train_y, num_classes = 10)\n",
    "test_y = keras.utils.to_categorical(test_y, num_classes = 10)\n",
    "print (train_y.shape)\n",
    "print (test_y.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### DepthwiseConv2D\n",
    "```py\n",
    "class DepthwiseConv2D(Conv2D):\n",
    "    \"\"\"Depthwise separable 2D convolution.\n",
    "    Depthwise Separable convolutions consist in performing\n",
    "    just the first step in a depthwise spatial convolution\n",
    "    (which acts on each input channel separately).\n",
    "    It does not perform the pointwise convolution (second step).\n",
    "    The `depth_multiplier` argument controls how many\n",
    "    output channels are generated per input channel in the depthwise step.\n",
    "    # Arguments\n",
    "        kernel_size: An integer or tuple/list of 2 integers, specifying the\n",
    "            width and height of the 2D convolution window.\n",
    "            Can be a single integer to specify the same value for\n",
    "            all spatial dimensions.\n",
    "        strides: An integer or tuple/list of 2 integers,\n",
    "            specifying the strides of the convolution along the width and height.\n",
    "            Can be a single integer to specify the same value for\n",
    "            all spatial dimensions.\n",
    "            Specifying any stride value != 1 is incompatible with specifying\n",
    "            any `dilation_rate` value != 1.\n",
    "        padding: one of `\"valid\"` or `\"same\"` (case-insensitive).\n",
    "        depth_multiplier: The number of depthwise convolution output channels\n",
    "            for each input channel.\n",
    "            The total number of depthwise convolution output\n",
    "            channels will be equal to `filterss_in * depth_multiplier`.\n",
    "        data_format: A string,\n",
    "            one of `channels_last` (default) or `channels_first`.\n",
    "            The ordering of the dimensions in the inputs.\n",
    "            `channels_last` corresponds to inputs with shape\n",
    "            `(batch, height, width, channels)` while `channels_first`\n",
    "            corresponds to inputs with shape\n",
    "            `(batch, channels, height, width)`.\n",
    "            It defaults to the `image_data_format` value found in your\n",
    "            Keras config file at `~/.keras/keras.json`.\n",
    "            If you never set it, then it will be \"channels_last\".\n",
    "        activation: Activation function to use\n",
    "            (see [activations](../activations.md)).\n",
    "            If you don't specify anything, no activation is applied\n",
    "            (ie. \"linear\" activation: `a(x) = x`).\n",
    "        use_bias: Boolean, whether the layer uses a bias vector.\n",
    "        depthwise_initializer: Initializer for the depthwise kernel matrix\n",
    "            (see [initializers](../initializers.md)).\n",
    "        bias_initializer: Initializer for the bias vector\n",
    "            (see [initializers](../initializers.md)).\n",
    "        depthwise_regularizer: Regularizer function applied to\n",
    "            the depthwise kernel matrix\n",
    "            (see [regularizer](../regularizers.md)).\n",
    "        bias_regularizer: Regularizer function applied to the bias vector\n",
    "            (see [regularizer](../regularizers.md)).\n",
    "        activity_regularizer: Regularizer function applied to\n",
    "            the output of the layer (its \"activation\").\n",
    "            (see [regularizer](../regularizers.md)).\n",
    "        depthwise_constraint: Constraint function applied to\n",
    "            the depthwise kernel matrix\n",
    "            (see [constraints](../constraints.md)).\n",
    "        bias_constraint: Constraint function applied to the bias vector\n",
    "            (see [constraints](../constraints.md)).\n",
    "    # Input shape\n",
    "        4D tensor with shape:\n",
    "        `(batch, channels, rows, cols)` if data_format='channels_first'\n",
    "        or 4D tensor with shape:\n",
    "        `(batch, rows, cols, channels)` if data_format='channels_last'.\n",
    "    # Output shape\n",
    "        4D tensor with shape:\n",
    "        `(batch, filters, new_rows, new_cols)` if data_format='channels_first'\n",
    "        or 4D tensor with shape:\n",
    "        `(batch, new_rows, new_cols, filters)` if data_format='channels_last'.\n",
    "        `rows` and `cols` values might have changed due to padding.\n",
    "    \"\"\"\n",
    "```"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "def MobileNetV1(input_shape, k):\n",
    "    inputs = Input(shape=input_shape)\n",
    "    x = Conv2D(filters=32, kernel_size=(3,3), strides=(2,2), padding='same', activation='relu')(inputs)\n",
    "    x = DepthwiseConv2D(kernel_size=(3,3),padding='same', activation='relu',)(x)\n",
    "    x = Conv2D(filters=64, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "    x = DepthwiseConv2D(kernel_size=(3,3), padding='same', activation='relu',)(x)\n",
    "    x = Conv2D(filters=128, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "    x = DepthwiseConv2D(kernel_size=(3,3), strides=(2,2),padding='same', activation='relu',)(x)\n",
    "    x = Conv2D(filters=128, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "    x = DepthwiseConv2D(kernel_size=(3,3),padding='same', activation='relu',)(x)\n",
    "    x = Conv2D(filters=256, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "    x = DepthwiseConv2D(kernel_size=(3,3), strides=(2,2),padding='same', activation='relu',)(x)\n",
    "    x = Conv2D(filters=256, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "    x = DepthwiseConv2D(kernel_size=(3,3),padding='same', activation='relu',)(x)\n",
    "    x = Conv2D(filters=512, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "    x = DepthwiseConv2D(kernel_size=(3,3), strides=(2,2),padding='same', activation='relu',)(x)\n",
    "    x = Conv2D(filters=512, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "    x = DepthwiseConv2D(kernel_size=(3,3), strides=(2,2),padding='same', activation='relu',)(x)\n",
    "    x = Conv2D(filters=1024, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "    x = DepthwiseConv2D(kernel_size=(3,3),padding='same', activation='relu',)(x)\n",
    "    x = Conv2D(filters=1024, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "    x = GlobalAveragePooling2D()(x)\n",
    "    x = Dense(1024, activation='relu')(x)\n",
    "    x = Dense(k, activation='softmax')(x)\n",
    "    model = Model(inputs, x)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def NaiveConvNet(input_shape, k):\n",
    "    inputs = Input(shape=input_shape)\n",
    "    x = Conv2D(filters=32, kernel_size=(3,3), strides=(2,2), padding='same', activation='relu')(inputs)\n",
    "    x = Conv2D(filters=64, kernel_size=(3,3),padding='same', activation='relu')(x)\n",
    "    x = Conv2D(filters=128, kernel_size=(3,3),padding='same', activation='relu')(x)\n",
    "    x = Conv2D(filters=128, kernel_size=(3,3), strides=(2,2),padding='same', activation='relu')(x)\n",
    "    x = Conv2D(filters=256, kernel_size=(3,3),padding='same', activation='relu')(x)\n",
    "    x = Conv2D(filters=256, kernel_size=(3,3), strides=(2,2),padding='same', activation='relu')(x)\n",
    "    x = Conv2D(filters=512, kernel_size=(3,3),padding='same', activation='relu')(x)\n",
    "    x = Conv2D(filters=512, kernel_size=(3,3), strides=(2,2),padding='same', activation='relu')(x)\n",
    "    x = Conv2D(filters=1024, kernel_size=(3,3), strides=(2,2),padding='same', activation='relu')(x)\n",
    "    x = Conv2D(filters=1024, kernel_size=(3,3),padding='same', activation='relu')(x)\n",
    "    x = GlobalAveragePooling2D()(x)\n",
    "    x = Dense(1024, activation='relu')(x)\n",
    "    x = Dense(k, activation='softmax')(x)\n",
    "    model = Model(inputs, x)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_3 (InputLayer)         (None, 224, 224, 3)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_12 (Conv2D)           (None, 112, 112, 32)      896       \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_1 (Depthwis (None, 112, 112, 32)      320       \n",
      "_________________________________________________________________\n",
      "conv2d_13 (Conv2D)           (None, 112, 112, 64)      2112      \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_2 (Depthwis (None, 112, 112, 64)      640       \n",
      "_________________________________________________________________\n",
      "conv2d_14 (Conv2D)           (None, 112, 112, 128)     8320      \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_3 (Depthwis (None, 56, 56, 128)       1280      \n",
      "_________________________________________________________________\n",
      "conv2d_15 (Conv2D)           (None, 56, 56, 128)       16512     \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_4 (Depthwis (None, 56, 56, 128)       1280      \n",
      "_________________________________________________________________\n",
      "conv2d_16 (Conv2D)           (None, 56, 56, 256)       33024     \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_5 (Depthwis (None, 28, 28, 256)       2560      \n",
      "_________________________________________________________________\n",
      "conv2d_17 (Conv2D)           (None, 28, 28, 256)       65792     \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_6 (Depthwis (None, 28, 28, 256)       2560      \n",
      "_________________________________________________________________\n",
      "conv2d_18 (Conv2D)           (None, 28, 28, 512)       131584    \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_7 (Depthwis (None, 14, 14, 512)       5120      \n",
      "_________________________________________________________________\n",
      "conv2d_19 (Conv2D)           (None, 14, 14, 512)       262656    \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_8 (Depthwis (None, 7, 7, 512)         5120      \n",
      "_________________________________________________________________\n",
      "conv2d_20 (Conv2D)           (None, 7, 7, 1024)        525312    \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_9 (Depthwis (None, 7, 7, 1024)        10240     \n",
      "_________________________________________________________________\n",
      "conv2d_21 (Conv2D)           (None, 7, 7, 1024)        1049600   \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d_2 ( (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_3 (Dense)              (None, 1024)              1049600   \n",
      "_________________________________________________________________\n",
      "dense_4 (Dense)              (None, 1000)              1025000   \n",
      "=================================================================\n",
      "Total params: 4,199,528\n",
      "Trainable params: 4,199,528\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "MobileNet_v1_model = MobileNetV1((224,224,3), 1000)\n",
    "MobileNet_v1_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_4 (InputLayer)         (None, 224, 224, 3)       0         \n",
      "_________________________________________________________________\n",
      "conv2d_22 (Conv2D)           (None, 112, 112, 32)      896       \n",
      "_________________________________________________________________\n",
      "conv2d_23 (Conv2D)           (None, 112, 112, 64)      18496     \n",
      "_________________________________________________________________\n",
      "conv2d_24 (Conv2D)           (None, 112, 112, 128)     73856     \n",
      "_________________________________________________________________\n",
      "conv2d_25 (Conv2D)           (None, 56, 56, 128)       147584    \n",
      "_________________________________________________________________\n",
      "conv2d_26 (Conv2D)           (None, 56, 56, 256)       295168    \n",
      "_________________________________________________________________\n",
      "conv2d_27 (Conv2D)           (None, 28, 28, 256)       590080    \n",
      "_________________________________________________________________\n",
      "conv2d_28 (Conv2D)           (None, 28, 28, 512)       1180160   \n",
      "_________________________________________________________________\n",
      "conv2d_29 (Conv2D)           (None, 14, 14, 512)       2359808   \n",
      "_________________________________________________________________\n",
      "conv2d_30 (Conv2D)           (None, 7, 7, 1024)        4719616   \n",
      "_________________________________________________________________\n",
      "conv2d_31 (Conv2D)           (None, 7, 7, 1024)        9438208   \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d_3 ( (None, 1024)              0         \n",
      "_________________________________________________________________\n",
      "dense_5 (Dense)              (None, 1024)              1049600   \n",
      "_________________________________________________________________\n",
      "dense_6 (Dense)              (None, 1000)              1025000   \n",
      "=================================================================\n",
      "Total params: 20,898,472\n",
      "Trainable params: 20,898,472\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "NaiveConvNet_model = NaiveConvNet((224,224,3), 1000)\n",
    "NaiveConvNet_model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Simple_MobileNetV1(input_shape, k):\n",
    "    inputs = Input(shape=input_shape)\n",
    "    x = Conv2D(filters=32, kernel_size=(3,3), strides=(2,2), padding='same', activation='relu')(inputs)\n",
    "    x = DepthwiseConv2D(kernel_size=(3,3),padding='same', activation='relu',)(x)\n",
    "    x = Conv2D(filters=64, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "    x = DepthwiseConv2D(kernel_size=(3,3), padding='same', activation='relu',)(x)\n",
    "    x = Conv2D(filters=128, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "    x = DepthwiseConv2D(kernel_size=(3,3), strides=(2,2),padding='same', activation='relu',)(x)\n",
    "    x = Conv2D(filters=128, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "    x = DepthwiseConv2D(kernel_size=(3,3),padding='same', activation='relu',)(x)\n",
    "#     x = Conv2D(filters=256, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "#     x = DepthwiseConv2D(kernel_size=(3,3), strides=(2,2),padding='same', activation='relu',)(x)\n",
    "#     x = Conv2D(filters=256, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "# #     x = DepthwiseConv2D(kernel_size=(3,3),padding='same', activation='relu',)(x)\n",
    "# #     x = Conv2D(filters=512, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "# #     x = DepthwiseConv2D(kernel_size=(3,3), strides=(2,2),padding='same', activation='relu',)(x)\n",
    "# #     x = Conv2D(filters=512, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "# #     x = DepthwiseConv2D(kernel_size=(3,3), strides=(2,2),padding='same', activation='relu',)(x)\n",
    "# #     x = Conv2D(filters=1024, kernel_size=(1,1),padding='same', activation='relu')(x)\n",
    "# #     x = DepthwiseConv2D(kernel_size=(3,3),padding='same', activation='relu',)(x)\n",
    "    x = Conv2D(filters=128, kernel_size=(1,1), padding='same', activation='relu')(x)\n",
    "    x = GlobalAveragePooling2D()(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dense(128, activation='relu')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dense(k, activation='softmax')(x)\n",
    "    model = Model(inputs, x)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_6 (InputLayer)         (None, 28, 28, 1)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_37 (Conv2D)           (None, 14, 14, 32)        320       \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_14 (Depthwi (None, 14, 14, 32)        320       \n",
      "_________________________________________________________________\n",
      "conv2d_38 (Conv2D)           (None, 14, 14, 64)        2112      \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_15 (Depthwi (None, 14, 14, 64)        640       \n",
      "_________________________________________________________________\n",
      "conv2d_39 (Conv2D)           (None, 14, 14, 128)       8320      \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_16 (Depthwi (None, 7, 7, 128)         1280      \n",
      "_________________________________________________________________\n",
      "conv2d_40 (Conv2D)           (None, 7, 7, 128)         16512     \n",
      "_________________________________________________________________\n",
      "depthwise_conv2d_17 (Depthwi (None, 7, 7, 128)         1280      \n",
      "_________________________________________________________________\n",
      "conv2d_41 (Conv2D)           (None, 7, 7, 128)         16512     \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d_5 ( (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_3 (Batch (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "dense_9 (Dense)              (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "batch_normalization_4 (Batch (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "dense_10 (Dense)             (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 66,122\n",
      "Trainable params: 65,610\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n",
      "Epoch 1/20\n",
      "60000/60000 [==============================] - 48s 800us/step - loss: 0.5188 - acc: 0.8251\n",
      "Epoch 2/20\n",
      "60000/60000 [==============================] - 47s 778us/step - loss: 0.1834 - acc: 0.9452\n",
      "Epoch 3/20\n",
      "60000/60000 [==============================] - 46s 767us/step - loss: 0.1320 - acc: 0.9601\n",
      "Epoch 4/20\n",
      "60000/60000 [==============================] - 46s 766us/step - loss: 0.1172 - acc: 0.9654\n",
      "Epoch 5/20\n",
      "60000/60000 [==============================] - 46s 768us/step - loss: 0.1007 - acc: 0.9703\n",
      "Epoch 6/20\n",
      "60000/60000 [==============================] - 51s 845us/step - loss: 0.0936 - acc: 0.9726\n",
      "Epoch 7/20\n",
      "60000/60000 [==============================] - 50s 825us/step - loss: 0.0885 - acc: 0.9751\n",
      "Epoch 8/20\n",
      "60000/60000 [==============================] - 50s 834us/step - loss: 0.0840 - acc: 0.9762\n",
      "Epoch 9/20\n",
      "60000/60000 [==============================] - 51s 846us/step - loss: 0.0801 - acc: 0.9772\n",
      "Epoch 10/20\n",
      "60000/60000 [==============================] - 49s 818us/step - loss: 0.0810 - acc: 0.9776\n",
      "Epoch 11/20\n",
      "60000/60000 [==============================] - 50s 834us/step - loss: 0.0733 - acc: 0.9793\n",
      "Epoch 12/20\n",
      "60000/60000 [==============================] - 51s 853us/step - loss: 0.0701 - acc: 0.9802\n",
      "Epoch 13/20\n",
      "60000/60000 [==============================] - 51s 847us/step - loss: 0.0702 - acc: 0.9802\n",
      "Epoch 14/20\n",
      "60000/60000 [==============================] - 49s 817us/step - loss: 0.0663 - acc: 0.9815\n",
      "Epoch 15/20\n",
      "60000/60000 [==============================] - 49s 814us/step - loss: 0.0684 - acc: 0.9822\n",
      "Epoch 16/20\n",
      "60000/60000 [==============================] - 49s 823us/step - loss: 0.0665 - acc: 0.9825\n",
      "Epoch 17/20\n",
      "60000/60000 [==============================] - 51s 852us/step - loss: 0.0646 - acc: 0.9830\n",
      "Epoch 18/20\n",
      "60000/60000 [==============================] - 47s 785us/step - loss: 0.0610 - acc: 0.9838\n",
      "Epoch 19/20\n",
      "60000/60000 [==============================] - 48s 798us/step - loss: 0.0587 - acc: 0.9839\n",
      "Epoch 20/20\n",
      "60000/60000 [==============================] - 51s 846us/step - loss: 0.0566 - acc: 0.9851\n"
     ]
    }
   ],
   "source": [
    "Simple_MobileNetV1_model = Simple_MobileNetV1((28,28,1), 10)\n",
    "Simple_MobileNetV1_model.summary()\n",
    "Simple_MobileNetV1_model.compile(optimizer=optimizers.RMSprop(0.001), loss='categorical_crossentropy', metrics=['acc'])\n",
    "Simple_MobileNetV1_history = Simple_MobileNetV1_model.fit(train_x, train_y, epochs=20, batch_size=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "10000/10000 [==============================] - 1s 66us/step\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[0.0815806967404671, 0.9776]"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "Simple_MobileNetV1_model.evaluate(test_x, test_y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def Simple_NaiveConvNet(input_shape, k):\n",
    "    inputs = Input(shape=input_shape)\n",
    "    x = Conv2D(filters=32, kernel_size=(3,3), strides=(2,2), padding='same', activation='relu')(inputs)\n",
    "    x = Conv2D(filters=64, kernel_size=(3,3),padding='same', activation='relu')(x)\n",
    "    x = Conv2D(filters=128, kernel_size=(3,3),padding='same', activation='relu')(x)\n",
    "    x = Conv2D(filters=128, kernel_size=(3,3), strides=(2,2),padding='same', activation='relu')(x)\n",
    "#     x = Conv2D(filters=256, kernel_size=(3,3),padding='same', activation='relu')(x)\n",
    "#     x = Conv2D(filters=256, kernel_size=(3,3), strides=(2,2),padding='same', activation='relu')(x)\n",
    "#     x = Conv2D(filters=512, kernel_size=(3,3),padding='same', activation='relu')(x)\n",
    "#     x = Conv2D(filters=512, kernel_size=(3,3), strides=(2,2),padding='same', activation='relu')(x)\n",
    "#     x = Conv2D(filters=1024, kernel_size=(3,3), strides=(2,2),padding='same', activation='relu')(x)\n",
    "#     x = Conv2D(filters=1024, kernel_size=(3,3),padding='same', activation='relu')(x)\n",
    "    x = GlobalAveragePooling2D()(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dense(128, activation='relu')(x)\n",
    "    x = BatchNormalization()(x)\n",
    "    x = Dense(k, activation='softmax')(x)\n",
    "    model = Model(inputs, x)\n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "input_7 (InputLayer)         (None, 28, 28, 1)         0         \n",
      "_________________________________________________________________\n",
      "conv2d_42 (Conv2D)           (None, 14, 14, 32)        320       \n",
      "_________________________________________________________________\n",
      "conv2d_43 (Conv2D)           (None, 14, 14, 64)        18496     \n",
      "_________________________________________________________________\n",
      "conv2d_44 (Conv2D)           (None, 14, 14, 128)       73856     \n",
      "_________________________________________________________________\n",
      "conv2d_45 (Conv2D)           (None, 7, 7, 128)         147584    \n",
      "_________________________________________________________________\n",
      "global_average_pooling2d_6 ( (None, 128)               0         \n",
      "_________________________________________________________________\n",
      "batch_normalization_5 (Batch (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "dense_11 (Dense)             (None, 128)               16512     \n",
      "_________________________________________________________________\n",
      "batch_normalization_6 (Batch (None, 128)               512       \n",
      "_________________________________________________________________\n",
      "dense_12 (Dense)             (None, 10)                1290      \n",
      "=================================================================\n",
      "Total params: 259,082\n",
      "Trainable params: 258,570\n",
      "Non-trainable params: 512\n",
      "_________________________________________________________________\n",
      "Epoch 1/50\n",
      "60000/60000 [==============================] - 40s 664us/step - loss: 0.2388 - acc: 0.9250\n",
      "Epoch 2/50\n",
      "60000/60000 [==============================] - 40s 662us/step - loss: 0.1058 - acc: 0.9693\n",
      "Epoch 3/50\n",
      "60000/60000 [==============================] - 40s 659us/step - loss: 0.0862 - acc: 0.9758\n",
      "Epoch 4/50\n",
      "60000/60000 [==============================] - 43s 709us/step - loss: 0.0736 - acc: 0.9790\n",
      "Epoch 5/50\n",
      "60000/60000 [==============================] - 41s 676us/step - loss: 0.0662 - acc: 0.9808\n",
      "Epoch 6/50\n",
      "60000/60000 [==============================] - 39s 653us/step - loss: 0.0572 - acc: 0.9842\n",
      "Epoch 7/50\n",
      "60000/60000 [==============================] - 40s 659us/step - loss: 0.0542 - acc: 0.9849\n",
      "Epoch 8/50\n",
      "60000/60000 [==============================] - 41s 684us/step - loss: 0.0478 - acc: 0.9870\n",
      "Epoch 9/50\n",
      "60000/60000 [==============================] - 39s 652us/step - loss: 0.0453 - acc: 0.9885\n",
      "Epoch 10/50\n",
      "60000/60000 [==============================] - 39s 653us/step - loss: 0.0433 - acc: 0.9885\n",
      "Epoch 11/50\n",
      "60000/60000 [==============================] - 39s 653us/step - loss: 0.0408 - acc: 0.9892\n",
      "Epoch 12/50\n",
      "60000/60000 [==============================] - 39s 653us/step - loss: 0.0397 - acc: 0.9895\n",
      "Epoch 13/50\n",
      "60000/60000 [==============================] - 39s 650us/step - loss: 0.0345 - acc: 0.9905\n",
      "Epoch 14/50\n",
      "60000/60000 [==============================] - 39s 649us/step - loss: 0.0378 - acc: 0.9902\n",
      "Epoch 15/50\n",
      "60000/60000 [==============================] - 39s 645us/step - loss: 0.0334 - acc: 0.9912\n",
      "Epoch 16/50\n",
      "60000/60000 [==============================] - 39s 646us/step - loss: 0.0341 - acc: 0.9912\n",
      "Epoch 17/50\n",
      "60000/60000 [==============================] - 39s 646us/step - loss: 0.0296 - acc: 0.9919\n",
      "Epoch 18/50\n",
      "60000/60000 [==============================] - 39s 645us/step - loss: 0.0308 - acc: 0.9923\n",
      "Epoch 19/50\n",
      "60000/60000 [==============================] - 39s 645us/step - loss: 0.0317 - acc: 0.9918\n",
      "Epoch 20/50\n",
      "60000/60000 [==============================] - 39s 645us/step - loss: 0.0269 - acc: 0.9934\n",
      "Epoch 21/50\n",
      "60000/60000 [==============================] - 39s 644us/step - loss: 0.0286 - acc: 0.9928\n",
      "Epoch 22/50\n",
      "60000/60000 [==============================] - 39s 646us/step - loss: 0.0270 - acc: 0.9933\n",
      "Epoch 23/50\n",
      "60000/60000 [==============================] - 39s 654us/step - loss: 0.0273 - acc: 0.9933\n",
      "Epoch 24/50\n",
      "60000/60000 [==============================] - 39s 645us/step - loss: 0.0223 - acc: 0.9939\n",
      "Epoch 25/50\n",
      "60000/60000 [==============================] - 39s 646us/step - loss: 0.0242 - acc: 0.9936\n",
      "Epoch 26/50\n",
      "60000/60000 [==============================] - 39s 646us/step - loss: 0.0249 - acc: 0.9944\n",
      "Epoch 27/50\n",
      "60000/60000 [==============================] - 39s 648us/step - loss: 0.0224 - acc: 0.9944\n",
      "Epoch 28/50\n",
      "60000/60000 [==============================] - 39s 648us/step - loss: 0.0242 - acc: 0.9939\n",
      "Epoch 29/50\n",
      "60000/60000 [==============================] - 39s 649us/step - loss: 0.0244 - acc: 0.9940\n",
      "Epoch 30/50\n",
      "60000/60000 [==============================] - 40s 659us/step - loss: 0.0208 - acc: 0.9946\n",
      "Epoch 31/50\n",
      "60000/60000 [==============================] - 39s 654us/step - loss: 0.0232 - acc: 0.9945\n",
      "Epoch 32/50\n",
      "60000/60000 [==============================] - 41s 686us/step - loss: 0.0188 - acc: 0.9952\n",
      "Epoch 33/50\n",
      "60000/60000 [==============================] - 42s 706us/step - loss: 0.0216 - acc: 0.9949\n",
      "Epoch 34/50\n",
      "60000/60000 [==============================] - 43s 709us/step - loss: 0.0222 - acc: 0.9944\n",
      "Epoch 35/50\n",
      "60000/60000 [==============================] - 43s 721us/step - loss: 0.0183 - acc: 0.9958\n",
      "Epoch 36/50\n",
      "60000/60000 [==============================] - 45s 747us/step - loss: 0.0182 - acc: 0.9953\n",
      "Epoch 37/50\n",
      "60000/60000 [==============================] - 44s 741us/step - loss: 0.0198 - acc: 0.9953\n",
      "Epoch 38/50\n",
      "60000/60000 [==============================] - 44s 727us/step - loss: 0.0185 - acc: 0.9955\n",
      "Epoch 39/50\n",
      "60000/60000 [==============================] - 44s 732us/step - loss: 0.0187 - acc: 0.9956\n",
      "Epoch 40/50\n",
      "60000/60000 [==============================] - 42s 708us/step - loss: 0.0160 - acc: 0.9963\n",
      "Epoch 41/50\n",
      "60000/60000 [==============================] - 41s 686us/step - loss: 0.0153 - acc: 0.9965\n",
      "Epoch 42/50\n",
      "60000/60000 [==============================] - 44s 730us/step - loss: 0.0168 - acc: 0.9961\n",
      "Epoch 43/50\n",
      "60000/60000 [==============================] - 44s 737us/step - loss: 0.0206 - acc: 0.9955\n",
      "Epoch 44/50\n",
      "60000/60000 [==============================] - 44s 737us/step - loss: 0.0225 - acc: 0.9956\n",
      "Epoch 45/50\n",
      "60000/60000 [==============================] - 44s 735us/step - loss: 0.0166 - acc: 0.9960\n",
      "Epoch 46/50\n",
      "60000/60000 [==============================] - 44s 741us/step - loss: 0.0165 - acc: 0.9964\n",
      "Epoch 47/50\n",
      "60000/60000 [==============================] - 44s 737us/step - loss: 0.0136 - acc: 0.9970\n",
      "Epoch 48/50\n",
      "60000/60000 [==============================] - 45s 742us/step - loss: 0.0172 - acc: 0.9962\n",
      "Epoch 49/50\n",
      "60000/60000 [==============================] - 44s 741us/step - loss: 0.0142 - acc: 0.9967\n",
      "Epoch 50/50\n",
      "60000/60000 [==============================] - 45s 746us/step - loss: 0.0161 - acc: 0.9966\n"
     ]
    }
   ],
   "source": [
    "Simple_NaiveConvNet_model = Simple_NaiveConvNet((28,28,1), 10)\n",
    "Simple_NaiveConvNet_model.summary()\n",
    "Simple_NaiveConvNet_model.compile(optimizer=optimizers.RMSprop(0.001), loss='categorical_crossentropy', metrics=['acc'])\n",
    "Simple_NaiveConvNet_history = Simple_NaiveConvNet_model.fit(train_x, train_y, epochs=50, batch_size=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
